
# If using Google Colab...
# Mount my Google Drive

from google.colab import drive
drive.mount('/content/drive')

Mounted at /content/drive


# Import necessary libraries

import re, string, unicodedata                          # Import Regex, string and unicodedata.
import wordcloud                                        # Import wordcloud to create word clouds
from wordcloud import WordCloud
import contractions                                     # Import contractions library.
from bs4 import BeautifulSoup                           # Import BeautifulSoup.

import pandas as pd                                     # Import Pandas for data
import numpy as np                                      # Import numpy.
import pandas as pd                                     # Import pandas.
import nltk                                             # Import Natural Language Tool-Kit

nltk.download('stopwords')                              # Download Stopwords.
nltk.download('punkt')
nltk.download('wordnet')

from nltk.corpus import stopwords                       # Import stopwords.
from nltk.tokenize import word_tokenize, sent_tokenize  # Import Tokenizer.
from nltk.stem.wordnet import WordNetLemmatizer         # Import Lemmatizer.

from sklearn.feature_extraction.text import CountVectorizer          #For Bag of words
from sklearn.feature_extraction.text import TfidfVectorizer          #For TF-IDF

from sklearn.ensemble import RandomForestClassifier     # Import Random Forest classifier
from sklearn.model_selection import train_test_split    # Import Train Test Split for modeling fitting
from sklearn.model_selection import cross_val_score     # Import cross validation score
from sklearn.model_selection import RandomizedSearchCV  # Import RandomizedSearchCV for hyperparameter tuning
from sklearn.datasets import load_digits    

from collections import Counter                         # Import Counter to count word frequency

from scipy.stats import randint as sp_randint

from sklearn import metrics                             # Import metrics for evaluating the model
from sklearn.metrics import confusion_matrix            # Import confusion matrix for evaluating the model

import matplotlib.pyplot as plt                         # Import plt for visualization
import seaborn as sns                                   # Import seaborn for visualization

[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Unzipping corpora/wordnet.zip.


# Load the data

df = pd.read_csv('/content/drive/My Drive/Colab Notebooks/NLP/Tweets.csv')


# Check the shape of the data

df.shape

(14640, 15)


# Check the first 5 rows of data

df.head()


# Show the column names

df.columns

Index(['tweet_id', 'airline_sentiment', 'airline_sentiment_confidence',
       'negativereason', 'negativereason_confidence', 'airline',
       'airline_sentiment_gold', 'name', 'negativereason_gold',
       'retweet_count', 'text', 'tweet_coord', 'tweet_created',
       'tweet_location', 'user_timezone'],
      dtype='object')


# Show the description of the data

df.describe()


# Check how many null values there are in each column

df.isna().sum()

tweet_id                            0
airline_sentiment                   0
airline_sentiment_confidence        0
negativereason                   5462
negativereason_confidence        4118
airline                             0
airline_sentiment_gold          14600
name                                0
negativereason_gold             14608
retweet_count                       0
text                                0
tweet_coord                     13621
tweet_created                       0
tweet_location                   4733
user_timezone                    4820
dtype: int64


# Show the percentage of values in each column which are null

(df.isna().sum() * 100 / df.index.size).round(2)

tweet_id                         0.00
airline_sentiment                0.00
airline_sentiment_confidence     0.00
negativereason                  37.31
negativereason_confidence       28.13
airline                          0.00
airline_sentiment_gold          99.73
name                             0.00
negativereason_gold             99.78
retweet_count                    0.00
text                             0.00
tweet_coord                     93.04
tweet_created                    0.00
tweet_location                  32.33
user_timezone                   32.92
dtype: float64


# Count the number of each sentiment

df['airline_sentiment'].value_counts()

negative    9178
neutral     3099
positive    2363
Name: airline_sentiment, dtype: int64


# Visualize the distribution of sentiment across all tweets

plt.figure(figsize=(3,5))
sns.countplot(
    x = 'airline_sentiment', 
    data = df, 
    order = df.airline_sentiment.value_counts().index,
    palette = 'viridis')

plt.show()


# Count the distribution of all tweets among each airline

df['airline'].value_counts()

United            3822
US Airways        2913
American          2759
Southwest         2420
Delta             2222
Virgin America     504
Name: airline, dtype: int64


# Plot the distribution of all tweets among each airline


plt.figure(figsize=(15,5))
sns.countplot(
    x = 'airline', 
    data = df, 
    order = df.airline.value_counts().index, 
    palette = 'viridis')

plt.show()


# Plot the distribution of Sentiment of tweets for each airline

g = sns.FacetGrid(
    data = df, 
    col = 'airline', 
    col_wrap = 3, 
    height = 5, 
    aspect = 0.7
) 

g = g.map(
    sns.countplot, 
    'airline_sentiment',
    order = df.airline_sentiment.value_counts().index,
    palette = 'viridis'
) 

plt.show()


# Count the distribution of all negative reasons

df['negativereason'].value_counts()

Customer Service Issue         2910
Late Flight                    1665
Can't Tell                     1190
Cancelled Flight                847
Lost Luggage                    724
Bad Flight                      580
Flight Booking Problems         529
Flight Attendant Complaints     481
longlines                       178
Damaged Luggage                  74
Name: negativereason, dtype: int64


# Plot the distribution of all the negative reasons

plt.figure(figsize=(15,5))
sns.countplot(
    x = 'negativereason', 
    data = df, 
    order = df.negativereason.value_counts().index, 
    palette = 'viridis')

plt.xticks(rotation=45)

plt.show()


# Plot the word cloud graph of tweets for positive and negative sentiment separately.


# Define a function to create a word cloud of the tweet text

df_copy = df.copy()

df_copy['text'] = df_copy.apply(lambda row: nltk.word_tokenize(row['text']), axis=1) # Tokenization of data

def show_wordcloud(df_copy, title):
    text = ' '.join(df_copy['text'].astype(str).tolist())                 # Converting Summary column into list
    stopwords = set(wordcloud.STOPWORDS)                                  # instantiate the stopwords from wordcloud
    
    fig_wordcloud = wordcloud.WordCloud(stopwords=stopwords,background_color='white',          # Setting the different parameter of stopwords
                    colormap='viridis', width=800, height=600).generate(text)
    
    plt.figure(figsize=(14,11), frameon=True)                             
    plt.imshow(fig_wordcloud)  
    plt.axis('off')
    plt.title(title, fontsize=30)
    plt.show()


# Plot a word cloud of text for positive tweets

show_wordcloud(df_copy[df_copy['airline_sentiment'] == 'positive'], title = "Positive Sentiment Tweets")


# Plot a word cloud of text for negative tweets

show_wordcloud(df_copy[df_copy['airline_sentiment'] == 'negative'], title = "Negative Sentiment Tweets")


# Drop all other columns except “text” and “airline_sentiment”

df.drop(['tweet_id', 'airline_sentiment_confidence',
       'negativereason', 'negativereason_confidence', 'airline',
       'airline_sentiment_gold', 'name', 'negativereason_gold',
       'retweet_count', 'tweet_coord', 'tweet_created',
       'tweet_location', 'user_timezone'], axis=1, inplace=True)


# Check the shape of the data now that we've dropped unnecessary columns

df.shape

(14640, 2)


# Print the first 5 rows of data

df.head()


# Display full dataframe information so full Tweet text displays
pd.set_option('display.max_colwidth', None)

# Check the first 5 rows of data
df.head()


# Define a function to strip html tags from the text
def strip_html(text):
    soup = BeautifulSoup(text, "html.parser")                    
    return soup.get_text()

# Apply the function to strip html tags from the text
df['text'] = df['text'].apply(lambda x: strip_html(x))

# Check the first 10 rows to see how the data has changed
df.head(10)


# Define function to remove http links using Regular Expression.
for i, row in df.iterrows():
    clean_tweet = re.sub(r"http\S+", "", df.at[i, 'text'])
    df.at[i,'text'] = clean_tweet

# Check the first 10 rows to see if it removed URLs
df.head(10)


# Define a function to replay contractions in the text with their two word forms
def replace_contractions(text):
    """Replace contractions in string of text"""
    return contractions.fix(text)

# Apply the function to remove contractions from the tweet text
df['text'] = df['text'].apply(lambda x: replace_contractions(x))

# Check the first 5 rows to see how the data has changed
df.head()


# Show first 25 tweets to find some with numbers in them
df.head(25)


# Define a function to remove numbers from the text of each tweet
def remove_numbers(text):
  text = re.sub(r'\d+', '', text)
  return text

# Apply the function to remove numbers from the tweet text
df['text'] = df['text'].apply(lambda x: remove_numbers(x))

# Check the first 25 rows to see how the data has changed
df.head(25)


# Apply a lambda function with nltk's word_tokenize to tokenize the tweet text
df['text'] = df.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)

# Check the first 5 rows to see how the data has changed
df.head()


# Define a function to remove non-ASCII characters
def remove_non_ascii(words):
    """Remove non-ASCII characters from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')
        new_words.append(new_word)
    return new_words

# Apply the function to remove non-ASCII characters from the tweet text
df['text'] = df['text'].apply(lambda x: remove_non_ascii(x))

# Check the first 25 rows to see how the data has changed
df.head(25)


# Define a function to remove punctuation from the text
def remove_punctuation(words):
    """Remove punctuation from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = re.sub(r'[^\w\s]', '', word)
        if new_word != '':
            new_words.append(new_word)
    return new_words

# Apply the function to remove punctuation from the tweet text
df['text'] = df['text'].apply(lambda x: remove_punctuation(x))

# Check the first 5 rows to see how the data has changed
df.head()


# Define variable to use English stopwords and keep the word "not" bc it's important for sentiment analysis
stopwords = stopwords.words('english')
stopwords.remove('not')

# Define a function to remove stop words from the text
def remove_stopwords(words):
    """Remove stop words from list of tokenized words"""
    new_words = []                        # Create empty list to store pre-processed words.
    for word in words:
        if word not in stopwords:
            new_words.append(word)        # Append processed words to new list.
    return new_words

# Apply the function to remove stop words from the tweet text
df['text'] = df['text'].apply(lambda x: remove_stopwords(x))

# Check the first 5 rows to see how the data has changed
df.head()


# Define a function to convert all words to lowercase
def to_lowercase(words):
    """Convert all characters to lowercase from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = word.lower()
        new_words.append(new_word)
    return new_words

# Apply the function to remove non-ASCII characters from the tweet text
df['text'] = df['text'].apply(lambda x: to_lowercase(x))

# Check the first 5 rows to see how the data has changed
df.head()


# Define a function to lemmatize the words in the tweets
def lemmatize_list(words):
    """Lemmatize verbs in list of tokenized words"""
    lemmatizer = WordNetLemmatizer()
    lemmas = []                           # Create empty list to store pre-processed words.
    for word in words:
        lemma = lemmatizer.lemmatize(word)
        lemmas.append(lemma)              # Append processed words to new list.
    return lemmas

# Apply the function to lemmatize the words in each tweet
df['text'] = df['text'].apply(lambda x: lemmatize_list(x))

# Check the first 5 rows to see how the data has changed
df.head()


# Aggregate the preprocessing functions into one
def normalize(words):
    words = remove_non_ascii(words)
    words = remove_punctuation(words)
    words = remove_stopwords(words)
    words = to_lowercase(words)
    words = lemmatize_list(words)
    return ' '.join(words)

# Apply the aggregated function and join the words in the list to convert back to text string in df
df['text'] = df.apply(lambda row: normalize(row['text']), axis=1)

# Check the first 5 rows
df.head()


# Create a copy of the dataframe
df_copy = df.copy()

# Tokenize the text column in the df copy so we can count word frequency
df_copy['text'] = df_copy.apply(lambda row: nltk.word_tokenize(row['text']), axis=1) # Tokenization of data

# Define a variable using the Counter() function
top = Counter([item for sublist in df_copy['text'] for item in sublist])


# Show the frequency of the words in the text column
freq = pd.DataFrame(top.most_common())
freq.columns = ['Common_words','count']
freq.style.background_gradient(cmap='Reds')


# Create CountVectorizer vectorizer
# Keep only 1000 features 
cv = CountVectorizer(max_features=1000) 

# Fit and transform on the text column
cv_features = cv.fit_transform(df['text'])

# Convert the features to an array
cv_features = cv_features.toarray()


# Check the shape of the new array
cv_features.shape

(14640, 1000)


# Create TF-IDF vectorizer 
# Keep only 2000 features
tfidf = TfidfVectorizer(max_features=1000)

# Fit and transform on the text column
tf_features = tfidf.fit_transform(df['text'])

# Convert the features to an array
tf_features = tf_features.toarray()


# Check the shape of the new array
tf_features.shape

(14640, 1000)


labels = df['airline_sentiment']


# Split the CountVectorizer data into training and testing set.

X_train_cv, X_test_cv, y_train_cv, y_test_cv = train_test_split(cv_features, labels, test_size=0.3, random_state=12)

# Check the shape of the split data
print("X_train")
print(X_train_cv.shape)
print("X_test")
print(X_test_cv.shape)
print("y_train")
print(y_train_cv.shape)
print("y_test")
print(X_test_cv.shape)

X_train
(10248, 1000)
X_test
(4392, 1000)
y_train
(10248,)
y_test
(4392, 1000)


# Using Random Forest to build model for the classification of reviews
rf = RandomForestClassifier(n_jobs=-1)

# Fit to the CountVectorizer data
cv_rf = rf.fit(X_train_cv, y_train_cv)

print(cv_rf)

RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
                       criterion='gini', max_depth=None, max_features='auto',
                       max_leaf_nodes=None, max_samples=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, n_estimators=100,
                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,
                       warm_start=False)


# Calculate the cross validation score on training
print(np.mean(cross_val_score(cv_rf, cv_features, labels, cv=10)))

0.7224726775956285


%%time

# Parameters to pass in RandomizedSearchCV
param_dist = {'n_estimators': sp_randint(4,200),
              "max_depth": [3, None],
              "max_features": sp_randint(1, 11),
              "min_samples_split": sp_randint(2, 11),
              "min_samples_leaf": sp_randint(1, 11),
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

# Type of scoring used to compare parameter combinations
#scorer = metrics.make_scorer(metrics.recall_score)

# Random search of parameters, using 10 fold cross validation, 
# Search across 100 different combinations, and use all available cores
cv_rf_random = RandomizedSearchCV(
    estimator = rf, 
    param_distributions = param_dist, 
    n_iter = 100, 
    cv = 3, 
    random_state=12)

# Fit the random search model
cv_rf_random.fit(X_train_cv, y_train_cv)

print("Best parameters are {} with CV score={}:" .format(cv_rf_random.best_params_,cv_rf_random.best_score_))

Best parameters are {'bootstrap': False, 'criterion': 'gini', 'max_depth': None, 'max_features': 7, 'min_samples_leaf': 1, 'min_samples_split': 7, 'n_estimators': 108} with CV score=0.7663934426229507:
CPU times: user 1min 45s, sys: 7.72 s, total: 1min 53s
Wall time: 6min 59s


# Training the tuned model
cv_rf_tuned = RandomForestClassifier(
    bootstrap = False,
    criterion = 'gini',
    max_depth = None, 
    max_features = 7, 
    min_samples_leaf = 1,
    min_samples_split = 7,
    n_estimators = 108,
    n_jobs = -1
    )

# Fit the tuned model
cv_rf_tuned.fit(X_train_cv, y_train_cv)

RandomForestClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,
                       criterion='gini', max_depth=None, max_features=7,
                       max_leaf_nodes=None, max_samples=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=7,
                       min_weight_fraction_leaf=0.0, n_estimators=108,
                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,
                       warm_start=False)


# Saving the prediction on train data as a result
cv_train_result =  cv_rf_tuned.predict(X_train_cv)  

# Saving the prediction on test data as a result
cv_test_result =  cv_rf_tuned.predict(X_test_cv)  

# Calculate the accuracy score
print("Accuracy on training set: ", cv_rf_tuned.score(X_train_cv, y_train_cv))
print("Accuracy on test set: ", cv_rf_tuned.score(X_test_cv, y_test_cv))

#Recall of the model
print("Recall on training set: ", metrics.recall_score(y_train_cv , cv_train_result, average = 'macro'))
print("Recall on test set: ", metrics.recall_score(y_test_cv , cv_test_result, average = 'macro'))

# Calculate the F1 score 
print("F1 score on training set: ", metrics.f1_score(y_train_cv, cv_train_result, average='macro'))
print("F1 score on test set: ", metrics.f1_score(y_test_cv, cv_test_result, average='macro'))

Accuracy on training set:  0.9877049180327869
Accuracy on test set:  0.7780054644808743
Recall on training set:  0.9827662320389144
Recall on test set:  0.6552485912469046
F1 score on training set:  0.9822087221194025
F1 score on test set:  0.6867705377436425


# Print and plot a Confusion matrix

conf_mat = confusion_matrix(y_test_cv, cv_test_result)

print(conf_mat)
print()

df_cm = pd.DataFrame(conf_mat, index = [i for i in ["Actual - Negative", "Actual - Neutral", "Actual - Positive"]],
                  columns = [i for i in ["Predicted - Negative", "Predicted - Neutral", "Predicted - Positive"]])
plt.figure(figsize = (10,7))
sns.heatmap(df_cm, annot=True, fmt='g')
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
plt.show()

[[2594  143   47]
 [ 401  411   72]
 [ 228   84  412]]


# Show the top 40 features for CountVectorizer & Random Forest Model

all_features = cv.get_feature_names()              #Instantiate the feature from the vectorizer
top_features=''                                    # Addition of top 40 feature into top_feature after training the model
feat=cv_rf_tuned.feature_importances_
features=np.argsort(feat)[::-1]
for i in features[0:40]:
    top_features+=all_features[i]
    top_features+=' '

print("The Top 40 features in this model are:")
top_features

The Top 40 features in this model are:

'thank thanks not hour usairways jetblue great cancelled flight delayed southwestair united customer love bag hold awesome americanair delay virginamerica best time get amazing call service hr plane still day dm good worst fleek never much airline phone min need '


wordcloud_cv = WordCloud(background_color="white",colormap='viridis',width=2000, 
                          height=1000).generate(top_features)

# Display the generated image:
plt.imshow(wordcloud_cv, interpolation='bilinear')
plt.figure(1, figsize=(14, 11), frameon='equal')
plt.title('Top 40 Features CountVectorizer WordCloud', fontsize=20)
plt.axis("off")
plt.show()


# Split the TF-IDF data into training and testing set.

X_train_tf, X_test_tf, y_train_tf, y_test_tf = train_test_split(tf_features, labels, test_size=0.3, random_state=12)

# Check the shape of the split data
print("X_train_tf")
print(X_train_tf.shape)
print("X_test_tf")
print(X_test_tf.shape)
print("y_train_tf")
print(y_train_tf.shape)
print("y_test_tf")
print(X_test_tf.shape)

X_train_tf
(10248, 1000)
X_test_tf
(4392, 1000)
y_train_tf
(10248,)
y_test_tf
(4392, 1000)


# Using Random Forest to build model for the classification of reviews
rf2 = RandomForestClassifier(n_jobs=-1)

# Fit to the TF-IDF data
tf_rf = rf2.fit(X_train_tf, y_train_tf)

print(tf_rf)

RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
                       criterion='gini', max_depth=None, max_features='auto',
                       max_leaf_nodes=None, max_samples=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, n_estimators=100,
                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,
                       warm_start=False)


# Calculate the cross validation score on training
print(np.mean(cross_val_score(tf_rf, tf_features, labels, cv=10)))

0.7172814207650274


%%time

# Parameters to pass in RandomizedSearchCV
param_dist2 = {'n_estimators': sp_randint(4,200),
              "max_depth": [3, None],
              "max_features": sp_randint(1, 11),
              "min_samples_split": sp_randint(2, 11),
              "min_samples_leaf": sp_randint(1, 11),
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

# Type of scoring used to compare parameter combinations
#scorer = metrics.make_scorer(metrics.recall_score)

# Random search of parameters, using 10 fold cross validation, 
# Search across 100 different combinations, and use all available cores
tf_rf_random = RandomizedSearchCV(
    estimator = rf2, 
    param_distributions = param_dist2, 
    n_iter = 100, 
    cv = 3, 
    random_state=12)

# Fit the random search model
tf_rf_random.fit(X_train_tf, y_train_tf)

print("Best parameters are {} with CV score={}:" .format(tf_rf_random.best_params_,tf_rf_random.best_score_))

Best parameters are {'bootstrap': False, 'criterion': 'entropy', 'max_depth': None, 'max_features': 5, 'min_samples_leaf': 1, 'min_samples_split': 10, 'n_estimators': 119} with CV score=0.762392661982826:
CPU times: user 1min 49s, sys: 7.86 s, total: 1min 57s
Wall time: 8min 2s


# Training the tuned model
tf_rf_tuned = RandomForestClassifier(
    bootstrap = False,
    criterion = 'entropy',
    max_depth = None, 
    max_features = 5, 
    min_samples_leaf = 1,
    min_samples_split = 10,
    n_estimators = 119,
    n_jobs = -1
    )

# Fit the tuned model
tf_rf_tuned.fit(X_train_tf, y_train_tf)

RandomForestClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,
                       criterion='entropy', max_depth=None, max_features=5,
                       max_leaf_nodes=None, max_samples=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=10,
                       min_weight_fraction_leaf=0.0, n_estimators=119,
                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,
                       warm_start=False)


# Saving the prediction on train data as a result
tf_train_result =  tf_rf_tuned.predict(X_train_tf)  

# Saving the prediction on test data as a result
tf_test_result =  tf_rf_tuned.predict(X_test_tf)  

# Calculate the accuracy score
print("Accuracy on training set: ", tf_rf_tuned.score(X_train_tf, y_train_tf))
print("Accuracy on test set: ", cv_rf_tuned.score(X_test_tf, y_test_tf))

#Recall of the model
print("Recall on training set: ", metrics.recall_score(y_train_tf , tf_train_result, average = 'macro'))
print("Recall on test set: ", metrics.recall_score(y_test_tf , tf_test_result, average = 'macro'))

# Calculate the F1 score 
print("F1 score on training set: ", metrics.f1_score(y_train_tf, tf_train_result, average='macro'))
print("F1 score on test set: ", metrics.f1_score(y_test_tf, tf_test_result, average='macro'))

Accuracy on training set:  0.9893637782982045
Accuracy on test set:  0.31397996357012753
Recall on training set:  0.9856492853009379
Recall on test set:  0.6309952554107604
F1 score on training set:  0.9842361382975451
F1 score on test set:  0.6685496892714712


# Print and plot a Confusion matrix

conf_mat2 = confusion_matrix(y_test_tf, tf_test_result)

print(conf_mat2)
print()

df_cm2 = pd.DataFrame(conf_mat2, index = [i for i in ["Actual - Negative", "Actual - Neutral", "Actual - Positive"]],
                  columns = [i for i in ["Predicted - Negative", "Predicted - Neutral", "Predicted - Positive"]])
plt.figure(figsize = (10,7))
sns.heatmap(df_cm2, annot=True, fmt='g')
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
plt.show()

[[2645  111   28]
 [ 439  372   73]
 [ 269   77  378]]


# Show the top 40 features for TF-IDF & Random Forest Model

all_features2 = tfidf.get_feature_names()              #Instantiate the feature from the vectorizer
top_features2=''                                    # Addition of top 40 feature into top_feature after training the model
feat2=tf_rf_tuned.feature_importances_
features2=np.argsort(feat2)[::-1]
for i in features2[0:40]:
    top_features2+=all_features2[i]
    top_features2+=' '

print("The Top 40 features in this model are:")
top_features2

The Top 40 features in this model are:

'thank thanks not jetblue southwestair usairways united hour flight americanair great virginamerica delayed cancelled hold love customer get bag awesome service delay best hr need time call plane please much help good amazing still would airline worst dm got day '


wordcloud_tf = WordCloud(background_color="white",colormap='viridis',width=2000, 
                          height=1000).generate(top_features2)

# Display the generated image:
plt.imshow(wordcloud_tf, interpolation='bilinear')
plt.figure(1, figsize=(14, 11), frameon='equal')
plt.title('Top 40 Features TF-IDF WordCloud', fontsize=20)
plt.axis("off")
plt.show()

	tweet_id	airline_sentiment	airline_sentiment_confidence	negativereason	negativereason_confidence	airline	airline_sentiment_gold	name	negativereason_gold	text	tweet_coord	tweet_created	tweet_location	user_timezone
0	570306133677760513	neutral	1.0000	NaN	NaN	Virgin America	NaN	cairdin	NaN	@VirginAmerica What @dhepburn said.	NaN	2015-02-24 11:35:52 -0800	NaN	Eastern Time (US & Canada)
1	570301130888122368	positive	0.3486	NaN	0.0000	Virgin America	NaN	jnardino	NaN	@VirginAmerica plus you've added commercials t...	NaN	2015-02-24 11:15:59 -0800	NaN	Pacific Time (US & Canada)
2	570301083672813571	neutral	0.6837	NaN	NaN	Virgin America	NaN	yvonnalynn	NaN	@VirginAmerica I didn't today... Must mean I n...	NaN	2015-02-24 11:15:48 -0800	Lets Play	Central Time (US & Canada)
3	570301031407624196	negative	1.0000	Bad Flight	0.7033	Virgin America	NaN	jnardino	NaN	@VirginAmerica it's really aggressive to blast...	NaN	2015-02-24 11:15:36 -0800	NaN	Pacific Time (US & Canada)
4	570300817074462722	negative	1.0000	Can't Tell	1.0000	Virgin America	NaN	jnardino	NaN	@VirginAmerica and it's a really big bad thing...	NaN	2015-02-24 11:14:45 -0800	NaN	Pacific Time (US & Canada)

	tweet_id	airline_sentiment_confidence	negativereason_confidence	retweet_count
count	1.464000e+04	14640.000000	10522.000000	14640.000000
mean	5.692184e+17	0.900169	0.638298	0.082650
std	7.791112e+14	0.162830	0.330440	0.745778
min	5.675883e+17	0.335000	0.000000	0.000000
25%	5.685592e+17	0.692300	0.360600	0.000000
50%	5.694779e+17	1.000000	0.670600	0.000000
75%	5.698905e+17	1.000000	1.000000	0.000000
max	5.703106e+17	1.000000	1.000000	44.000000

	airline_sentiment	text
0	neutral	[@, VirginAmerica, What, @, dhepburn, said, .]
1	positive	[@, VirginAmerica, plus, you, have, added, commercials, to, the, experience, ..., tacky, .]
2	neutral	[@, VirginAmerica, I, did, not, today, ..., Must, mean, I, need, to, take, another, trip, !]
3	negative	[@, VirginAmerica, it, is, really, aggressive, to, blast, obnoxious, ``, entertainment, '', in, your, guests, ', faces, &, they, have, little, recourse]
4	negative	[@, VirginAmerica, and, it, is, a, really, big, bad, thing, about, it]

	airline_sentiment	text
0	neutral	[VirginAmerica, What, dhepburn, said]
1	positive	[VirginAmerica, plus, you, have, added, commercials, to, the, experience, tacky]
2	neutral	[VirginAmerica, I, did, not, today, Must, mean, I, need, to, take, another, trip]
3	negative	[VirginAmerica, it, is, really, aggressive, to, blast, obnoxious, entertainment, in, your, guests, faces, they, have, little, recourse]
4	negative	[VirginAmerica, and, it, is, a, really, big, bad, thing, about, it]

	airline_sentiment	text
0	neutral	virginamerica dhepburn said
1	positive	virginamerica plus added commercial experience tacky
2	neutral	virginamerica not today must mean need take another trip
3	negative	virginamerica really aggressive blast obnoxious entertainment guest face little recourse
4	negative	virginamerica really big bad thing

	airline_sentiment	text
0	neutral	@VirginAmerica What @dhepburn said.
1	positive	@VirginAmerica plus you have added commercials to the experience... tacky.
2	neutral	@VirginAmerica I did not today... Must mean I need to take another trip!
3	negative	@VirginAmerica it is really aggressive to blast obnoxious "entertainment" in your guests' faces & they have little recourse
4	negative	@VirginAmerica and it is a really big bad thing about it
5	negative	@VirginAmerica seriously would pay $30 a flight for seats that did not have this playing.\nit is really the only bad thing about flying VA
6	positive	@VirginAmerica yes, nearly every time I fly VX this “ear worm” will not go away :)
7	neutral	@VirginAmerica Really missed a prime opportunity for Men Without Hats parody, there.
8	positive	@virginamerica Well, I did not…but NOW I DO! :-D
9	positive	@VirginAmerica it was amazing, and arrived an hour early. you are too good to me.
10	neutral	@VirginAmerica did you know that suicide is the second leading because of death among teens 10-24
11	positive	@VirginAmerica I <3 pretty graphics. so much better than minimal iconography. :D
12	positive	@VirginAmerica This is such a great deal! Already thinking about my 2nd trip to @Australia & I have not even gone on my 1st trip yet! ;p
13	positive	@VirginAmerica @virginmedia I am flying your #fabulous #Seductive skies again! you take all the #stress away from travel
14	positive	@VirginAmerica Thanks!
15	negative	@VirginAmerica SFO-PDX schedule is still MIA.
16	positive	@VirginAmerica So excited for my first cross country flight LAX to MCO I have heard nothing but great things about Virgin America. #29DaysToGo
17	negative	@VirginAmerica I flew from NYC to SFO last week and could not fully sit in my seat due to two large gentleman on either side of me. HELP!
18	positive	I ❤️ flying @VirginAmerica. ☺️👍
19	positive	@VirginAmerica you know what would be amazingly awesome? BOS-FLL PLEASE!!!!!!! I want to fly with only you.
20	negative	@VirginAmerica why are your first fares in May over three times more than other carriers when all seats are available to select???
21	positive	@VirginAmerica I love this graphic.
22	positive	@VirginAmerica I love the hipster innovation. You are a feel good brand.
23	neutral	@VirginAmerica will you be making BOS>LAS non stop permanently anytime soon?
24	negative	@VirginAmerica you guys messed up my seating.. I reserved seating with my friends and you guys gave my seat away ... 😡 I want free internet

	airline_sentiment	text
0	neutral	@VirginAmerica What @dhepburn said.
1	positive	@VirginAmerica plus you have added commercials to the experience... tacky.
2	neutral	@VirginAmerica I did not today... Must mean I need to take another trip!
3	negative	@VirginAmerica it is really aggressive to blast obnoxious "entertainment" in your guests' faces & they have little recourse
4	negative	@VirginAmerica and it is a really big bad thing about it
5	negative	@VirginAmerica seriously would pay $ a flight for seats that did not have this playing.\nit is really the only bad thing about flying VA
6	positive	@VirginAmerica yes, nearly every time I fly VX this “ear worm” will not go away :)
7	neutral	@VirginAmerica Really missed a prime opportunity for Men Without Hats parody, there.
8	positive	@virginamerica Well, I did not…but NOW I DO! :-D
9	positive	@VirginAmerica it was amazing, and arrived an hour early. you are too good to me.
10	neutral	@VirginAmerica did you know that suicide is the second leading because of death among teens -
11	positive	@VirginAmerica I < pretty graphics. so much better than minimal iconography. :D
12	positive	@VirginAmerica This is such a great deal! Already thinking about my nd trip to @Australia & I have not even gone on my st trip yet! ;p
13	positive	@VirginAmerica @virginmedia I am flying your #fabulous #Seductive skies again! you take all the #stress away from travel
14	positive	@VirginAmerica Thanks!
15	negative	@VirginAmerica SFO-PDX schedule is still MIA.
16	positive	@VirginAmerica So excited for my first cross country flight LAX to MCO I have heard nothing but great things about Virgin America. #DaysToGo
17	negative	@VirginAmerica I flew from NYC to SFO last week and could not fully sit in my seat due to two large gentleman on either side of me. HELP!
18	positive	I ❤️ flying @VirginAmerica. ☺️👍
19	positive	@VirginAmerica you know what would be amazingly awesome? BOS-FLL PLEASE!!!!!!! I want to fly with only you.
20	negative	@VirginAmerica why are your first fares in May over three times more than other carriers when all seats are available to select???
21	positive	@VirginAmerica I love this graphic.
22	positive	@VirginAmerica I love the hipster innovation. You are a feel good brand.
23	neutral	@VirginAmerica will you be making BOS>LAS non stop permanently anytime soon?
24	negative	@VirginAmerica you guys messed up my seating.. I reserved seating with my friends and you guys gave my seat away ... 😡 I want free internet

	airline_sentiment	text
0	neutral	[@, VirginAmerica, What, @, dhepburn, said, .]
1	positive	[@, VirginAmerica, plus, you, have, added, commercials, to, the, experience, ..., tacky, .]
2	neutral	[@, VirginAmerica, I, did, not, today, ..., Must, mean, I, need, to, take, another, trip, !]
3	negative	[@, VirginAmerica, it, is, really, aggressive, to, blast, obnoxious, ``, entertainment, '', in, your, guests, ', faces, &, they, have, little, recourse]
4	negative	[@, VirginAmerica, and, it, is, a, really, big, bad, thing, about, it]
5	negative	[@, VirginAmerica, seriously, would, pay, $, a, flight, for, seats, that, did, not, have, this, playing, ., it, is, really, the, only, bad, thing, about, flying, VA]
6	positive	[@, VirginAmerica, yes, ,, nearly, every, time, I, fly, VX, this, , ear, worm, , will, not, go, away, :, )]
7	neutral	[@, VirginAmerica, Really, missed, a, prime, opportunity, for, Men, Without, Hats, parody, ,, there, .]
8	positive	[@, virginamerica, Well, ,, I, did, not...but, NOW, I, DO, !, :, -D]
9	positive	[@, VirginAmerica, it, was, amazing, ,, and, arrived, an, hour, early, ., you, are, too, good, to, me, .]
10	neutral	[@, VirginAmerica, did, you, know, that, suicide, is, the, second, leading, because, of, death, among, teens, -]
11	positive	[@, VirginAmerica, I, <, pretty, graphics, ., so, much, better, than, minimal, iconography, ., :, D]
12	positive	[@, VirginAmerica, This, is, such, a, great, deal, !, Already, thinking, about, my, nd, trip, to, @, Australia, &, I, have, not, even, gone, on, my, st, trip, yet, !, ;, p]
13	positive	[@, VirginAmerica, @, virginmedia, I, am, flying, your, #, fabulous, #, Seductive, skies, again, !, you, take, all, the, #, stress, away, from, travel]
14	positive	[@, VirginAmerica, Thanks, !]
15	negative	[@, VirginAmerica, SFO-PDX, schedule, is, still, MIA, .]
16	positive	[@, VirginAmerica, So, excited, for, my, first, cross, country, flight, LAX, to, MCO, I, have, heard, nothing, but, great, things, about, Virgin, America, ., #, DaysToGo]
17	negative	[@, VirginAmerica, I, flew, from, NYC, to, SFO, last, week, and, could, not, fully, sit, in, my, seat, due, to, two, large, gentleman, on, either, side, of, me, ., HELP, !]
18	positive	[I, , flying, @, VirginAmerica, ., ]
19	positive	[@, VirginAmerica, you, know, what, would, be, amazingly, awesome, ?, BOS-FLL, PLEASE, !, !, !, !, !, !, !, I, want, to, fly, with, only, you, .]
20	negative	[@, VirginAmerica, why, are, your, first, fares, in, May, over, three, times, more, than, other, carriers, when, all, seats, are, available, to, select, ?, ?, ?]
21	positive	[@, VirginAmerica, I, love, this, graphic, .]
22	positive	[@, VirginAmerica, I, love, the, hipster, innovation, ., You, are, a, feel, good, brand, .]
23	neutral	[@, VirginAmerica, will, you, be, making, BOS, >, LAS, non, stop, permanently, anytime, soon, ?]
24	negative	[@, VirginAmerica, you, guys, messed, up, my, seating.., I, reserved, seating, with, my, friends, and, you, guys, gave, my, seat, away, ..., , I, want, free, internet]

	Common_words	count
0	flight	4568
1	united	4160
2	not	3901
3	usairways	3051
4	americanair	2961
5	southwestair	2461
6	jetblue	2395
7	get	1376
8	hour	1161
9	thanks	1079
10	cancelled	1056
11	service	991
12	time	946
13	customer	934
14	help	872
15	bag	766
16	u	746
17	would	739
18	plane	726
19	hold	643
20	need	633
21	thank	603
22	one	580
23	still	579
24	day	569
25	please	564
26	call	561
27	airline	556
28	gate	542
29	delayed	536
30	back	526
31	virginamerica	523
32	flightled	507
33	got	481
34	seat	480
35	phone	454
36	delay	441
37	like	432
38	today	430
39	hr	430

Natural Lanaguage Processing (NLP) Project¶

Data Description¶

1. Data Summary¶

Observations¶

Observations¶

Observations¶

Observations¶

2. Exploratory Data Analysis (EDA)¶

Observations¶

Distribution of Tweets by Airline¶

Observations¶

Distribution of Tweet Sentiment by Airline¶

Observations¶

Distribution of All Negative Reasons¶

Observations¶

Word Cloud Graphs of Tweets¶

Observations¶

Observations¶

3. Understanding of Data Columns¶

Observations¶

4. Data Pre-Processing¶

Noise Removal¶

Remove HTML Tags¶

Observations¶

Remove URLs¶

Observations¶

Replace Contractions¶

Observations¶

Remove Numbers¶

Observations¶

Tokenization¶

Observations¶

Normalization¶

Remove Special Characters¶

Observations¶

Remove Punctuation¶

Observations¶

Remove Stop Words¶

Observations¶

Convert to Lowercase¶

Observations¶

Lemmatization¶

Observations¶

Convert Back to Text String¶

Observations¶

5. Vectorization¶

Observation¶

Observations¶

CountVectorizer¶

Observations¶

TF-IDF Vectorizer¶

Observations¶

6. Modeling, Tuning, and Evaluation¶

Fit the CountVectorizer Model¶

Observations¶

Observations¶

Tune the CountVectorizer Model¶

Observations¶

Evaluate the CountVectorizer Model¶

Observations¶

Observations¶

Top 40 Features with CountVectorizer & Random Forest Model¶

Observations¶

Word Cloud with CountVectorizer & Random Forest Model¶

Observations¶

Fit the TF-IDF Model¶

Observations¶

Observations¶

Tune the TF-IDF Model¶

Observations¶

Evaluate the TF-IDF Model¶

Observations¶

Observations¶

Top 40 Features with TF-IDF¶

Observations¶

Word Cloud with TF-IDF¶

Observations¶

7. Conclusion¶